# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy.http import Request
from urllib import parse
from Article.items import ArticleItem
from Article.utils.common import get_md5

class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):

        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url": image_url},
                          callback=self.parse_detail)
            print(post_url)

        # 提取下一页
        next_urls = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_urls:
            yield Request(parse.urljoin(response.url, next_urls), callback=self.parse)


    def matchNumber(self, string):
        match_re = re.match(".*(\d+).*", string)
        if match_re:
            return match_re.group(1)
        else:
            return 0

    def parse_detail(self, response):

        articl_item = ArticleItem()
        front_image_url = response.meta.get("front_image_url", "")
        title = response.xpath('//div[@class="entry-header"]/h1/text()').extract_first("")
        praise_nums = self.matchNumber(
            response.xpath("//span[contains(@class,'vote-post-up')]/h10/text()").extract_first(""))
        # 收藏数目
        fav_nums = self.matchNumber(response.xpath("//span[contains(@class,'bookmark-btn')]/text()").extract_first(""))
        # 评论数目
        commont_nums = self.matchNumber(response.xpath("//a[@href='#article-comment']/span/text()").extract_first(""))

        createDate = response.xpath('//*[@id="post-111742"]/div[2]/p/text()').extract_first("").strip().replace(".", "")

        content = response.xpath("//div[@class='entry']").extract_first("")

        tag_list = response.xpath("//p[@class='entry-meta-hide-on-mobile']/a/text()").extract()
        tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
        # tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
        tags = ",".join(tag_list)

        articl_item['url_object_id'] = get_md5(response.url)
        articl_item['title'] = title
        articl_item['praise_nums'] = praise_nums
        articl_item['url'] = response.url
        articl_item['createDate'] = createDate
        articl_item['fav_nums'] = fav_nums
        articl_item['commont_nums'] = commont_nums
        articl_item['content'] = content
        articl_item['front_image_url'] = [front_image_url]
        articl_item['commont_nums'] = commont_nums
        articl_item['tags'] = tags

        yield articl_item